import shelve 
import hashlib
import os


newPath="/home/cyw/projects/function_sim_project/all_data/new_fam_lables.txt"
originPath="/home/cyw/projects/function_sim_project/all_data/fam_lables.txt"

def get_inf_from_txt(family_lable_Path):
    """
        从VT导出的家族信息中，获得样本和对应家族的标记
        部分样本存在重复，部分标记是未识别需要移除
    """
    cfNum=0
    errNum=0
    res={}
    with open(family_lable_Path) as file:
        for inf in file.readlines():
            temp=inf.split("	")
            name,lable=temp[0],temp[1].strip()
            if name in res:
                cfNum+=1
            # elif lable.split(":")[0]=="SINGLETON":
            #     errNum+=1
            #     continue
            else:
                res[name]=lable
    return res

def  get_use_sample_name():
    """
        获得被用于训练的样本名称
    """
    name=[]
    sampleLablesPath = "/home/cyw/projects/function_sim_project/all_data/pair_infs/sample_and_lables"
    with shelve.open(sampleLablesPath) as file:
        trainName,trainLable=file["trainName"],file["trainLable"]
        testName,testLable=file["testName"],file["testLable"]
        validName,validLable=file["validName"],file["validLable"]
    for i in trainName:
        name.append(i)
    for i in testName:
        name.append(i)
    for i in validName:
        name.append(i)
    return name 

def calulate_sample_md5(samplePath):
    """
        输入样本路径
        返回样本md5值
    """
    with open(samplePath, 'rb') as fp:
        data = fp.read()
    file_md5= hashlib.md5(data).hexdigest()
    return file_md5


def get_unfinished_sample_name():
    """
        处理原始样本和已处理样本，
        找到未处理样本名,
        
        获得的样本名重新获取一下json报告，如果还是存在样本无标记的话，应该就是md5值计算错误
    """
    sampleNamePath="/home/cyw/projects/data/functionsim/allorigindata/"
    for i,j,names in os.walk(sampleNamePath):
        print(len(names))
    tar1=get_inf_from_txt(newPath)
    res=[]
    for i in names:
        if i not in tar1:
            res.append(i)
    return res

if __name__=="__main__":
    tar1=get_inf_from_txt(newPath)
    tar2=get_inf_from_txt(originPath)
    lth1=len(tar1)
    lth2=len(tar2)
    print("原始样本数量：{}\n新标记的样本数量：{}".format(lth2,lth1))
    useedName=get_use_sample_name()
    print("之前实验所使用的样本量：{}".format(len(useedName)))

    # 新处理的样本个数是否完全
    # ----已经处理完全了------
    missNum=0
    missName=[]
    for i in useedName:
        if i not in tar1:
            missNum+=1
            missName.append(i)

    print("新处理的样本同之前相比，存在{}个未有标记".format(missNum))
    print(missName)
    print("样本的json文件存在，但是其家族标记没有输出,怀疑是之前的md5值计算错误")
    for i in missName:
        md5=calulate_sample_md5("/home/cyw/projects/data/functionsim/allorigindata/"+i)
        print(i,md5)
    print("确实存在样本MD5值计算错误。。。")
    # 不过也存在四个样本md5值计算一样的情况，看了一下
    # 12476aa0d72285be95f0c8c125a0bd36   6dcb616a8545920c8b128de2359c6f22  9ff91b23074ae660164df7f0b2faddea  2a8ea48b988d141fc33bb17ea5b0979e
    differNum=0
    for i in useedName:
        x=tar1.get(i,"None")
        y=tar2.get(i,"None")
        if x!=y or x==None:
            # print(x,y)
            differNum+=1
    print("共 {} 个样本的家族标记不同".format(differNum))

    # res=get_unfinished_sample_name()
    # print(res)
    # differNum=0
    
    # a={}
    # b={}
    # for i in res:
    #     md5=calulate_sample_md5("/home/cyw/projects/data/functionsim/allorigindata/"+i)
    #     print(i,md5)
    #     a[i]=True
    #     b[md5]=True
    # print("原始md5:{}                       实际md5:{}".format(len(a),len(b)))